Merge pull request #203 from cantino/data_output_agent

Data output agent

Andrew Cantino 10 years ago
parent
commit
25b62ae1f1

+ 1 - 0
CHANGES.md

@@ -1,5 +1,6 @@
1 1
 # Changes
2 2
 
3
+* 0.4 (April 10, 2014) - WebHooksController has been renamed to WebRequestsController and all HTTP verbs are now accepted and passed through to Agents' #receive_web_request method. The new DataOutputAgent returns JSON or RSS feeds of incoming Events via external web request.
3 4
 * 0.31 (Jan 2, 2014)   - Agents now have an optional keep\_events\_for option that is propagated to created events' expires\_at field, and they update their events' expires\_at fields on change.
4 5
 * 0.3 (Jan 1, 2014)    - Remove symbolization of memory, options, and payloads; convert memory, options, and payloads to JSON from YAML.  Migration will perform conversion and adjust tables to be UTF-8.  Recommend making a DB backup before migrating.
5 6
 * 0.2 (Nov 6, 2013)    - PeakDetectorAgent now uses `window_duration_in_days` and `min_peak_spacing_in_days`.  Additionally, peaks trigger when the time series rises over the standard deviation multiple, not after it starts to fall.

+ 1 - 1
Gemfile

@@ -50,12 +50,12 @@ platforms :ruby_18 do
50 50
 end
51 51
 
52 52
 group :development do
53
-  gem 'pry'
54 53
   gem 'binding_of_caller'
55 54
   gem 'better_errors'
56 55
 end
57 56
 
58 57
 group :development, :test do
58
+  gem 'pry'
59 59
   gem 'rspec-rails'
60 60
   gem 'rspec'
61 61
   gem 'shoulda-matchers'

+ 41 - 0
app/controllers/web_requests_controller.rb

@@ -0,0 +1,41 @@
1
+# This controller is designed to allow your Agents to receive cross-site Webhooks (POSTs), or to output data streams.
2
+# When a POST or GET is received, your Agent will have #receive_web_request called on itself with the incoming params,
3
+# method, and requested content-type.
4
+#
5
+# Requests are routed as follows:
6
+#   http://yourserver.com/users/:user_id/web_requests/:agent_id/:secret
7
+# where :user_id is a User's id, :agent_id is an Agent's id, and :secret is a token that should be user-specifiable in
8
+# an Agent that implements #receive_web_request. It is highly recommended that every Agent verify this token whenever
9
+# #receive_web_request is called. For example, one of your Agent's options could be :secret and you could compare this
10
+# value to params[:secret] whenever #receive_web_request is called on your Agent, rejecting invalid requests.
11
+#
12
+# Your Agent's #receive_web_request method should return an Array of json_or_string_response, status_code, and
13
+# optional mime type.  For example:
14
+#   [{status: "success"}, 200]
15
+# or
16
+#   ["not found", 404, 'text/plain']
17
+
18
+class WebRequestsController < ApplicationController
19
+  skip_before_filter :authenticate_user!
20
+
21
+  def handle_request
22
+    user = User.find_by_id(params[:user_id])
23
+    if user
24
+      agent = user.agents.find_by_id(params[:agent_id])
25
+      if agent
26
+        content, status, content_type = agent.trigger_web_request(params.except(:action, :controller, :agent_id, :user_id, :format), request.method_symbol.to_s, request.format.to_s)
27
+        if content.is_a?(String)
28
+          render :text => content, :status => status || 200, :content_type => content_type || 'text/plain'
29
+        elsif content.is_a?(Hash)
30
+          render :json => content, :status => status || 200
31
+        else
32
+          head(status || 200)
33
+        end
34
+      else
35
+        render :text => "agent not found", :status => 404
36
+      end
37
+    else
38
+      render :text => "user not found", :status => 404
39
+    end
40
+  end
41
+end

+ 0 - 39
app/controllers/webhooks_controller.rb

@@ -1,39 +0,0 @@
1
-# This controller is designed to allow your Agents to receive cross-site Webhooks (posts).  When POSTed, your Agent will
2
-# have #receive_webhook called on itself with the POST params.
3
-#
4
-# Make POSTs to the following URL:
5
-#   http://yourserver.com/users/:user_id/webhooks/:agent_id/:secret
6
-# where :user_id is your User's id, :agent_id is an Agent's id, and :secret is a token that should be
7
-# user-specifiable in your Agent.  It is highly recommended that you verify this token whenever #receive_webhook
8
-# is called.  For example, one of your Agent's options could be :secret and you could compare this value
9
-# to params[:secret] whenever #receive_webhook is called on your Agent, rejecting invalid requests.
10
-#
11
-# Your Agent's #receive_webhook method should return an Array of [json_or_string_response, status_code].  For example:
12
-#   [{status: "success"}, 200]
13
-# or
14
-#   ["not found", 404]
15
-
16
-class WebhooksController < ApplicationController
17
-  skip_before_filter :authenticate_user!
18
-
19
-  def create
20
-    user = User.find_by_id(params[:user_id])
21
-    if user
22
-      agent = user.agents.find_by_id(params[:agent_id])
23
-      if agent
24
-        response, status = agent.trigger_webhook(params.except(:action, :controller, :agent_id, :user_id))
25
-        if response.is_a?(String)
26
-          render :text => response, :status => status || 200
27
-        elsif response.is_a?(Hash)
28
-          render :json => response, :status => status || 200
29
-        else
30
-          head :ok
31
-        end
32
-      else
33
-        render :text => "agent not found", :status => :not_found
34
-      end
35
-    else
36
-      render :text => "user not found", :status => :not_found
37
-    end
38
-  end
39
-end

+ 13 - 5
app/models/agent.rb

@@ -73,7 +73,7 @@ class Agent < ActiveRecord::Base
73 73
     # Implement me in your subclass of Agent.
74 74
   end
75 75
 
76
-  def receive_webhook(params)
76
+  def receive_web_request(params, method, format)
77 77
     # Implement me in your subclass of Agent.
78 78
     ["not implemented", 404]
79 79
   end
@@ -136,10 +136,18 @@ class Agent < ActiveRecord::Base
136 136
     message.gsub(/<([^>]+)>/) { Utils.value_at(payload, $1) || "??" }
137 137
   end
138 138
 
139
-  def trigger_webhook(params)
140
-    receive_webhook(params).tap do
141
-      self.last_webhook_at = Time.now
142
-      save!
139
+  def trigger_web_request(params, method, format)
140
+    if respond_to?(:receive_webhook)
141
+      Rails.logger.warn "DEPRECATED: The .receive_webhook method is deprecated, please switch your Agent to use .receive_web_request."
142
+      receive_webhook(params).tap do
143
+        self.last_web_request_at = Time.now
144
+        save!
145
+      end
146
+    else
147
+      receive_web_request(params, method, format).tap do
148
+        self.last_web_request_at = Time.now
149
+        save!
150
+      end
143 151
     end
144 152
   end
145 153
 

+ 127 - 0
app/models/agents/data_output_agent.rb

@@ -0,0 +1,127 @@
1
+module Agents
2
+  class DataOutputAgent < Agent
3
+    cannot_be_scheduled!
4
+
5
+    description  do
6
+      <<-MD
7
+        The Agent outputs received events as either RSS or JSON.  Use it to output a public or private stream of Huginn data.
8
+
9
+        This Agent will output data at:
10
+
11
+        `https://#{ENV['DOMAIN']}/users/#{user.id}/web_requests/#{id || '<id>'}/:secret.xml`
12
+
13
+        where `:secret` is one of the allowed secrets specified in your options and the extension can be `xml` or `json`.
14
+
15
+        You can setup multiple secrets so that you can individually authorize external systems to
16
+        access your Huginn data.
17
+
18
+        Options:
19
+
20
+          * `secrets` - An array of tokens that the requestor must provide for light-weight authentication.
21
+          * `expected_receive_period_in_days` - How often you expect data to be received by this Agent from other Agents.
22
+          * `template` - A JSON object representing a mapping between item output keys and incoming event JSONPath values.  JSONPath values must start with `$`, or can be interpolated between `<` and `>` characters.  The `item` key will be repeated for every Event.
23
+      MD
24
+    end
25
+
26
+    def default_options
27
+      {
28
+        "secrets" => ["a-secret-key"],
29
+        "expected_receive_period_in_days" => 2,
30
+        "template" => {
31
+          "title" => "XKCD comics as a feed",
32
+          "description" => "This is a feed of recent XKCD comics, generated by Huginn",
33
+          "item" => {
34
+            "title" => "$.title",
35
+            "description" => "Secret hovertext: <$.hovertext>",
36
+            "link" => "$.url",
37
+          }
38
+        }
39
+      }
40
+    end
41
+
42
+    #"guid" => "",
43
+    #  "pubDate" => ""
44
+
45
+    def working?
46
+      last_receive_at && last_receive_at > options['expected_receive_period_in_days'].to_i.days.ago && !recent_error_logs?
47
+    end
48
+
49
+    def validate_options
50
+      unless options['secrets'].is_a?(Array) && options['secrets'].length > 0
51
+        errors.add(:base, "Please specify one or more secrets for 'authenticating' incoming feed requests")
52
+      end
53
+      unless options['expected_receive_period_in_days'].present? && options['expected_receive_period_in_days'].to_i > 0
54
+        errors.add(:base, "Please provide 'expected_receive_period_in_days' to indicate how many days can pass before this Agent is considered to be not working")
55
+      end
56
+
57
+      unless options['template'].present? && options['template']['item'].present? && options['template']['item'].is_a?(Hash)
58
+        errors.add(:base, "Please provide template and template.item")
59
+      end
60
+    end
61
+
62
+    def events_to_show
63
+      (options['events_to_show'].presence || 40).to_i
64
+    end
65
+
66
+    def feed_ttl
67
+      (options['ttl'].presence || 60).to_i
68
+    end
69
+
70
+    def feed_title
71
+      options['template']['title'].presence || "#{name} Event Feed"
72
+    end
73
+
74
+    def feed_description
75
+      options['template']['description'].presence || "A feed of Events received by the '#{name}' Huginn Agent"
76
+    end
77
+
78
+    def receive_web_request(params, method, format)
79
+      if options['secrets'].include?(params['secret'])
80
+        items = received_events.order('id desc').limit(events_to_show).map do |event|
81
+          interpolated = Utils.recursively_interpolate_jsonpaths(options['template']['item'], event.payload, :leading_dollarsign_is_jsonpath => true)
82
+          interpolated['guid'] = event.id
83
+          interpolated['pubDate'] = event.created_at.rfc2822.to_s
84
+          interpolated
85
+        end
86
+
87
+        if format =~ /json/
88
+          content = {
89
+            'title' => feed_title,
90
+            'description' => feed_description,
91
+            'pubDate' => Time.now,
92
+            'items' => items
93
+          }
94
+
95
+          return [content, 200]
96
+        else
97
+          content = Utils.unindent(<<-XML)
98
+            <?xml version="1.0" encoding="UTF-8" ?>
99
+            <rss version="2.0">
100
+            <channel>
101
+             <title>#{feed_title.encode(:xml => :text)}</title>
102
+             <description>#{feed_description.encode(:xml => :text)}</description>
103
+             <lastBuildDate>#{Time.now.rfc2822.to_s.encode(:xml => :text)}</lastBuildDate>
104
+             <pubDate>#{Time.now.rfc2822.to_s.encode(:xml => :text)}</pubDate>
105
+             <ttl>#{feed_ttl}</ttl>
106
+
107
+          XML
108
+
109
+          content += items.to_xml(:skip_types => true, :root => "items", :skip_instruct => true, :indent => 1).gsub(/^<\/?items>/, '').strip
110
+
111
+          content += Utils.unindent(<<-XML)
112
+            </channel>
113
+            </rss>
114
+          XML
115
+
116
+          return [content, 200, 'text/xml']
117
+        end
118
+      else
119
+        if format =~ /json/
120
+          return [{ :error => "Not Authorized" }, 401]
121
+        else
122
+          return ["Not Authorized", 401]
123
+        end
124
+      end
125
+    end
126
+  end
127
+end

+ 2 - 2
app/models/agents/twilio_agent.rb

@@ -75,10 +75,10 @@ module Agents
75 75
     end
76 76
 
77 77
     def post_url(server_url,secret)
78
-      "#{server_url}/users/#{self.user.id}/webhooks/#{self.id}/#{secret}"
78
+      "#{server_url}/users/#{self.user.id}/web_requests/#{self.id}/#{secret}"
79 79
     end
80 80
 
81
-    def receive_webhook(params)
81
+    def receive_web_request(params, method, format)
82 82
       if memory['pending_calls'].has_key? params['secret']
83 83
         response = Twilio::TwiML::Response.new {|r| r.Say memory['pending_calls'][params['secret']], :voice => 'woman'}
84 84
         memory['pending_calls'].delete params['secret']

+ 4 - 2
app/models/agents/webhook_agent.rb

@@ -1,6 +1,7 @@
1 1
 module Agents
2 2
   class WebhookAgent < Agent
3 3
     cannot_be_scheduled!
4
+    cannot_receive_events!
4 5
 
5 6
     description  do
6 7
         <<-MD
@@ -8,7 +9,7 @@ module Agents
8 9
 
9 10
         In order to create events with this agent, make a POST request to:
10 11
         ```
11
-           https://#{ENV['DOMAIN']}/users/#{user.id}/webhooks/#{id || '<id>'}/:secret
12
+           https://#{ENV['DOMAIN']}/users/#{user.id}/web_requests/#{id || '<id>'}/:secret
12 13
         ``` where `:secret` is specified in your options.
13 14
 
14 15
         The
@@ -36,8 +37,9 @@ module Agents
36 37
         "payload_path" => "payload"}
37 38
     end
38 39
 
39
-    def receive_webhook(params)
40
+    def receive_web_request(params, method, format)
40 41
       secret = params.delete('secret')
42
+      return ["Please use POST requests only", 401] unless method == "post"
41 43
       return ["Not Authorized", 401] unless secret == options['secret']
42 44
 
43 45
       create_event(:payload => payload_for(params))

+ 3 - 2
app/models/agents/website_agent.rb

@@ -59,8 +59,9 @@ module Agents
59 59
           'type' => "html",
60 60
           'mode' => "on_change",
61 61
           'extract' => {
62
-            'url' => {'css' => "#comic img", 'attr' => "src"},
63
-            'title' => {'css' => "#comic img", 'attr' => "title"}
62
+            'url' => { 'css' => "#comic img", 'attr' => "src" },
63
+            'title' => { 'css' => "#comic img", 'attr' => "alt" },
64
+            'hovertext' => { 'css' => "#comic img", 'attr' => "title" }
64 65
           }
65 66
       }
66 67
     end

+ 12 - 0
app/views/agents/agent_views/data_output_agent/_show.html.erb

@@ -0,0 +1,12 @@
1
+<p>
2
+  Data for this Agent is available at these URLs:
3
+</p>
4
+
5
+
6
+<ul>
7
+  <% @agent.options['secrets'].each do |secret| %>
8
+    <% url = lambda { |format| web_requests_url(:agent_id => @agent.id, :user_id => current_user.id, :secret => secret, :format => format) } %>
9
+    <li><%= link_to url.call(:json), url.call(:json), :target => :blank %></li>
10
+    <li><%= link_to url.call(:xml), url.call(:xml), :target => :blank %></li>
11
+  <% end %>
12
+</ul>

+ 3 - 0
app/views/agents/agent_views/webhook_agent/_show.html.erb

@@ -0,0 +1,3 @@
1
+<p>
2
+  Send WebHooks (POST requests) to this Agent at <%= link_to web_requests_url(:agent_id => @agent.id, :user_id => current_user.id, :secret => @agent.options['secret']), web_requests_url(:agent_id => @agent.id, :user_id => current_user.id, :secret => @agent.options['secret']), :target => :blank %>
3
+</p>

+ 3 - 1
config/routes.rb

@@ -31,7 +31,9 @@ Huginn::Application.routes.draw do
31 31
   match "/worker_status" => "worker_status#show"
32 32
 
33 33
   post "/users/:user_id/update_location/:secret" => "user_location_updates#create"
34
-  post "/users/:user_id/webhooks/:agent_id/:secret" => "webhooks#create"
34
+
35
+  match "/users/:user_id/web_requests/:agent_id/:secret" => "web_requests#handle_request", :as => :web_requests
36
+  post "/users/:user_id/webhooks/:agent_id/:secret" => "web_requests#handle_request" # legacy
35 37
 
36 38
 # To enable DelayedJobWeb, see the 'Enable DelayedJobWeb' section of the README.
37 39
 #  match "/delayed_job" => DelayedJobWeb, :anchor => false

+ 9 - 0
db/migrate/20140408150825_rename_webhook_to_web_request.rb

@@ -0,0 +1,9 @@
1
+class RenameWebhookToWebRequest < ActiveRecord::Migration
2
+  def up
3
+    rename_column :agents, :last_webhook_at, :last_web_request_at
4
+  end
5
+
6
+  def down
7
+    rename_column :agents, :last_web_request_at, :last_webhook_at
8
+  end
9
+end

+ 15 - 23
db/schema.rb

@@ -11,21 +11,21 @@
11 11
 #
12 12
 # It's strongly recommended to check this file into your version control system.
13 13
 
14
-ActiveRecord::Schema.define(:version => 20140216201250) do
14
+ActiveRecord::Schema.define(:version => 20140408150825) do
15 15
 
16 16
   create_table "agent_logs", :force => true do |t|
17
-    t.integer  "agent_id",                                             :null => false
18
-    t.text     "message",           :limit => 16777215,                :null => false
19
-    t.integer  "level",                                 :default => 3, :null => false
17
+    t.integer  "agent_id",                         :null => false
18
+    t.text     "message",                          :null => false
19
+    t.integer  "level",             :default => 3, :null => false
20 20
     t.integer  "inbound_event_id"
21 21
     t.integer  "outbound_event_id"
22
-    t.datetime "created_at",                                           :null => false
23
-    t.datetime "updated_at",                                           :null => false
22
+    t.datetime "created_at",                       :null => false
23
+    t.datetime "updated_at",                       :null => false
24 24
   end
25 25
 
26 26
   create_table "agents", :force => true do |t|
27 27
     t.integer  "user_id"
28
-    t.text     "options",               :limit => 16777215
28
+    t.text     "options"
29 29
     t.string   "type"
30 30
     t.string   "name"
31 31
     t.string   "schedule"
@@ -36,10 +36,10 @@ ActiveRecord::Schema.define(:version => 20140216201250) do
36 36
     t.datetime "created_at",                                                     :null => false
37 37
     t.datetime "updated_at",                                                     :null => false
38 38
     t.text     "memory",                :limit => 2147483647
39
-    t.datetime "last_webhook_at"
39
+    t.datetime "last_web_request_at"
40
+    t.integer  "keep_events_for",                             :default => 0,     :null => false
40 41
     t.datetime "last_event_at"
41 42
     t.datetime "last_error_log_at"
42
-    t.integer  "keep_events_for",                             :default => 0,     :null => false
43 43
     t.boolean  "propagate_immediately",                       :default => false, :null => false
44 44
   end
45 45
 
@@ -47,19 +47,11 @@ ActiveRecord::Schema.define(:version => 20140216201250) do
47 47
   add_index "agents", ["type"], :name => "index_agents_on_type"
48 48
   add_index "agents", ["user_id", "created_at"], :name => "index_agents_on_user_id_and_created_at"
49 49
 
50
-  create_table "contacts", :force => true do |t|
51
-    t.text     "message"
52
-    t.string   "name"
53
-    t.string   "email"
54
-    t.datetime "created_at", :null => false
55
-    t.datetime "updated_at", :null => false
56
-  end
57
-
58 50
   create_table "delayed_jobs", :force => true do |t|
59 51
     t.integer  "priority",                       :default => 0
60 52
     t.integer  "attempts",                       :default => 0
61 53
     t.text     "handler",    :limit => 16777215
62
-    t.text     "last_error", :limit => 16777215
54
+    t.text     "last_error"
63 55
     t.datetime "run_at"
64 56
     t.datetime "locked_at"
65 57
     t.datetime "failed_at"
@@ -74,11 +66,11 @@ ActiveRecord::Schema.define(:version => 20140216201250) do
74 66
   create_table "events", :force => true do |t|
75 67
     t.integer  "user_id"
76 68
     t.integer  "agent_id"
77
-    t.decimal  "lat",                              :precision => 15, :scale => 10
78
-    t.decimal  "lng",                              :precision => 15, :scale => 10
79
-    t.text     "payload",    :limit => 2147483647
80
-    t.datetime "created_at",                                                       :null => false
81
-    t.datetime "updated_at",                                                       :null => false
69
+    t.decimal  "lat",                            :precision => 15, :scale => 10
70
+    t.decimal  "lng",                            :precision => 15, :scale => 10
71
+    t.text     "payload",    :limit => 16777215
72
+    t.datetime "created_at",                                                     :null => false
73
+    t.datetime "updated_at",                                                     :null => false
82 74
     t.datetime "expires_at"
83 75
   end
84 76
 

+ 2 - 1
db/seeds.rb

@@ -32,7 +32,8 @@ unless user.agents.where(:name => "XKCD Source").exists?
32 32
                            'expected_update_period_in_days' => 5,
33 33
                            'extract' => {
34 34
                                'url' => { 'css' => "#comic img", 'attr' => "src" },
35
-                               'title' => { 'css' => "#comic img", 'attr' => "title" }
35
+                               'title' => { 'css' => "#comic img", 'attr' => "alt" },
36
+                               'hovertext' => { 'css' => "#comic img", 'attr' => "title" }
36 37
                            }
37 38
                        }).save!
38 39
 end

+ 12 - 8
lib/utils.rb

@@ -21,20 +21,24 @@ module Utils
21 21
     end
22 22
   end
23 23
 
24
-  def self.interpolate_jsonpaths(value, data)
25
-    value.gsub(/<[^>]+>/).each { |jsonpath|
26
-      Utils.values_at(data, jsonpath[1..-2]).first.to_s
27
-    }
24
+  def self.interpolate_jsonpaths(value, data, options = {})
25
+    if options[:leading_dollarsign_is_jsonpath] && value[0] == '$'
26
+      Utils.values_at(data, value).first.to_s
27
+    else
28
+      value.gsub(/<[^>]+>/).each { |jsonpath|
29
+        Utils.values_at(data, jsonpath[1..-2]).first.to_s
30
+      }
31
+    end
28 32
   end
29 33
 
30
-  def self.recursively_interpolate_jsonpaths(struct, data)
34
+  def self.recursively_interpolate_jsonpaths(struct, data, options = {})
31 35
     case struct
32 36
       when Hash
33
-        struct.inject({}) {|memo, (key, value)| memo[key] = recursively_interpolate_jsonpaths(value, data); memo }
37
+        struct.inject({}) {|memo, (key, value)| memo[key] = recursively_interpolate_jsonpaths(value, data, options); memo }
34 38
       when Array
35
-        struct.map {|elem| recursively_interpolate_jsonpaths(elem, data) }
39
+        struct.map {|elem| recursively_interpolate_jsonpaths(elem, data, options) }
36 40
       when String
37
-        interpolate_jsonpaths(struct, data)
41
+        interpolate_jsonpaths(struct, data, options)
38 42
       else
39 43
         struct
40 44
     end

+ 97 - 0
spec/controllers/web_requests_controller_spec.rb

@@ -0,0 +1,97 @@
1
+require 'spec_helper'
2
+
3
+describe WebRequestsController do
4
+  class Agents::WebRequestReceiverAgent < Agent
5
+    cannot_receive_events!
6
+    cannot_be_scheduled!
7
+
8
+    def receive_web_request(params, method, format)
9
+      if params.delete(:secret) == options[:secret]
10
+        memory[:web_request_values] = params
11
+        memory[:web_request_format] = format
12
+        memory[:web_request_method] = method
13
+        ["success", 200, memory['content_type']]
14
+      else
15
+        ["failure", 404]
16
+      end
17
+    end
18
+  end
19
+
20
+  before do
21
+    stub(Agents::WebRequestReceiverAgent).valid_type?("Agents::WebRequestReceiverAgent") { true }
22
+    @agent = Agents::WebRequestReceiverAgent.new(:name => "something", :options => { :secret => "my_secret" })
23
+    @agent.user = users(:bob)
24
+    @agent.save!
25
+  end
26
+
27
+  it "should not require login to receive a web request" do
28
+    @agent.last_web_request_at.should be_nil
29
+    post :handle_request, :user_id => users(:bob).to_param, :agent_id => @agent.id, :secret => "my_secret", :key => "value", :another_key => "5"
30
+    @agent.reload.last_web_request_at.should be_within(2).of(Time.now)
31
+    response.body.should == "success"
32
+    response.should be_success
33
+  end
34
+
35
+  it "should call receive_web_request" do
36
+    post :handle_request, :user_id => users(:bob).to_param, :agent_id => @agent.id, :secret => "my_secret", :key => "value", :another_key => "5"
37
+    @agent.reload
38
+    @agent.memory[:web_request_values].should == { 'key' => "value", 'another_key' => "5" }
39
+    @agent.memory[:web_request_format].should == "text/html"
40
+    @agent.memory[:web_request_method].should == "post"
41
+    response.body.should == "success"
42
+    response.headers['Content-Type'].should == 'text/plain; charset=utf-8'
43
+    response.should be_success
44
+
45
+    post :handle_request, :user_id => users(:bob).to_param, :agent_id => @agent.id, :secret => "not_my_secret", :no => "go"
46
+    @agent.reload.memory[:web_request_values].should_not == { 'no' => "go" }
47
+    response.body.should == "failure"
48
+    response.should be_missing
49
+  end
50
+
51
+  it "should accept gets" do
52
+    get :handle_request, :user_id => users(:bob).to_param, :agent_id => @agent.id, :secret => "my_secret", :key => "value", :another_key => "5"
53
+    @agent.reload
54
+    @agent.memory[:web_request_values].should == { 'key' => "value", 'another_key' => "5" }
55
+    @agent.memory[:web_request_format].should == "text/html"
56
+    @agent.memory[:web_request_method].should == "get"
57
+    response.body.should == "success"
58
+    response.should be_success
59
+  end
60
+
61
+  it "should pass through the received format" do
62
+    get :handle_request, :user_id => users(:bob).to_param, :agent_id => @agent.id, :secret => "my_secret", :key => "value", :another_key => "5", :format => :json
63
+    @agent.reload
64
+    @agent.memory[:web_request_values].should == { 'key' => "value", 'another_key' => "5" }
65
+    @agent.memory[:web_request_format].should == "application/json"
66
+    @agent.memory[:web_request_method].should == "get"
67
+
68
+    post :handle_request, :user_id => users(:bob).to_param, :agent_id => @agent.id, :secret => "my_secret", :key => "value", :another_key => "5", :format => :xml
69
+    @agent.reload
70
+    @agent.memory[:web_request_values].should == { 'key' => "value", 'another_key' => "5" }
71
+    @agent.memory[:web_request_format].should == "application/xml"
72
+    @agent.memory[:web_request_method].should == "post"
73
+
74
+    put :handle_request, :user_id => users(:bob).to_param, :agent_id => @agent.id, :secret => "my_secret", :key => "value", :another_key => "5", :format => :atom
75
+    @agent.reload
76
+    @agent.memory[:web_request_values].should == { 'key' => "value", 'another_key' => "5" }
77
+    @agent.memory[:web_request_format].should == "application/atom+xml"
78
+    @agent.memory[:web_request_method].should == "put"
79
+  end
80
+
81
+  it "can accept a content-type to return" do
82
+    @agent.memory['content_type'] = 'application/json'
83
+    @agent.save!
84
+    get :handle_request, :user_id => users(:bob).to_param, :agent_id => @agent.id, :secret => "my_secret", :key => "value", :another_key => "5"
85
+    response.headers['Content-Type'].should == 'application/json; charset=utf-8'
86
+  end
87
+
88
+  it "should fail on incorrect users" do
89
+    post :handle_request, :user_id => users(:jane).to_param, :agent_id => @agent.id, :secret => "my_secret", :no => "go"
90
+    response.should be_missing
91
+  end
92
+
93
+  it "should fail on incorrect agents" do
94
+    post :handle_request, :user_id => users(:bob).to_param, :agent_id => 454545, :secret => "my_secret", :no => "go"
95
+    response.should be_missing
96
+  end
97
+end

+ 0 - 54
spec/controllers/webhooks_controller_spec.rb

@@ -1,54 +0,0 @@
1
-require 'spec_helper'
2
-
3
-describe WebhooksController do
4
-  class Agents::WebhookReceiverAgent < Agent
5
-    cannot_receive_events!
6
-    cannot_be_scheduled!
7
-
8
-    def receive_webhook(params)
9
-      if params.delete(:secret) == options[:secret]
10
-        memory[:webhook_values] = params
11
-        ["success", 200]
12
-      else
13
-        ["failure", 404]
14
-      end
15
-    end
16
-  end
17
-
18
-  before do
19
-    stub(Agents::WebhookReceiverAgent).valid_type?("Agents::WebhookReceiverAgent") { true }
20
-    @agent = Agents::WebhookReceiverAgent.new(:name => "something", :options => { :secret => "my_secret" })
21
-    @agent.user = users(:bob)
22
-    @agent.save!
23
-  end
24
-
25
-  it "should not require login to trigger a webhook" do
26
-    @agent.last_webhook_at.should be_nil
27
-    post :create, :user_id => users(:bob).to_param, :agent_id => @agent.id, :secret => "my_secret", :key => "value", :another_key => "5"
28
-    @agent.reload.last_webhook_at.should be_within(2).of(Time.now)
29
-    response.body.should == "success"
30
-    response.should be_success
31
-  end
32
-
33
-  it "should call receive_webhook" do
34
-    post :create, :user_id => users(:bob).to_param, :agent_id => @agent.id, :secret => "my_secret", :key => "value", :another_key => "5"
35
-    @agent.reload.memory[:webhook_values].should == { 'key' => "value", 'another_key' => "5" }
36
-    response.body.should == "success"
37
-    response.should be_success
38
-
39
-    post :create, :user_id => users(:bob).to_param, :agent_id => @agent.id, :secret => "not_my_secret", :no => "go"
40
-    @agent.reload.memory[:webhook_values].should_not == { 'no' => "go" }
41
-    response.body.should == "failure"
42
-    response.should be_missing
43
-  end
44
-
45
-  it "should fail on incorrect users" do
46
-    post :create, :user_id => users(:jane).to_param, :agent_id => @agent.id, :secret => "my_secret", :no => "go"
47
-    response.should be_missing
48
-  end
49
-
50
-  it "should fail on incorrect agents" do
51
-    post :create, :user_id => users(:bob).to_param, :agent_id => 454545, :secret => "my_secret", :no => "go"
52
-    response.should be_missing
53
-  end
54
-end

+ 8 - 1
spec/lib/utils_spec.rb

@@ -28,8 +28,15 @@ describe Utils do
28 28
   end
29 29
 
30 30
   describe "#interpolate_jsonpaths" do
31
+    let(:payload) { { :there => { :world => "WORLD" }, :works => "should work" } }
32
+
31 33
     it "interpolates jsonpath expressions between matching <>'s" do
32
-      Utils.interpolate_jsonpaths("hello <$.there.world> this <escape works>", { :there => { :world => "WORLD" }, :works => "should work" }).should == "hello WORLD this should+work"
34
+      Utils.interpolate_jsonpaths("hello <$.there.world> this <escape works>", payload).should == "hello WORLD this should+work"
35
+    end
36
+
37
+    it "optionally supports treating values that start with '$' as raw JSONPath" do
38
+      Utils.interpolate_jsonpaths("$.there.world", payload).should == "$.there.world"
39
+      Utils.interpolate_jsonpaths("$.there.world", payload, :leading_dollarsign_is_jsonpath => true).should == "WORLD"
33 40
     end
34 41
   end
35 42
 

+ 48 - 0
spec/models/agent_spec.rb

@@ -514,7 +514,55 @@ describe Agent do
514 514
         end
515 515
       end
516 516
     end
517
+  end
518
+
519
+  describe ".trigger_web_request" do
520
+    class Agents::WebRequestReceiver < Agent
521
+      cannot_be_scheduled!
522
+    end
523
+
524
+    before do
525
+      stub(Agents::WebRequestReceiver).valid_type?("Agents::WebRequestReceiver") { true }
526
+    end
527
+
528
+    context "when .receive_web_request is defined" do
529
+      before do
530
+        @agent = Agents::WebRequestReceiver.new(:name => "something")
531
+        @agent.user = users(:bob)
532
+        @agent.save!
533
+
534
+        def @agent.receive_web_request(params, method, format)
535
+          memory['last_request'] = [params, method, format]
536
+          ['Ok!', 200]
537
+        end
538
+      end
539
+
540
+      it "calls the .receive_web_request hook, updates last_web_request_at, and saves" do
541
+        @agent.trigger_web_request({ :some_param => "some_value" }, "post", "text/html")
542
+        @agent.reload.memory['last_request'].should == [ { "some_param" => "some_value" }, "post", "text/html" ]
543
+        @agent.last_web_request_at.to_i.should be_within(1).of(Time.now.to_i)
544
+      end
545
+    end
517 546
 
547
+    context "when .receive_webhook is defined" do
548
+      before do
549
+        @agent = Agents::WebRequestReceiver.new(:name => "something")
550
+        @agent.user = users(:bob)
551
+        @agent.save!
552
+
553
+        def @agent.receive_webhook(params)
554
+          memory['last_webhook_request'] = params
555
+          ['Ok!', 200]
556
+        end
557
+      end
558
+
559
+      it "outputs a deprecation warning and calls .receive_webhook with the params" do
560
+        mock(Rails.logger).warn("DEPRECATED: The .receive_webhook method is deprecated, please switch your Agent to use .receive_web_request.")
561
+        @agent.trigger_web_request({ :some_param => "some_value" }, "post", "text/html")
562
+        @agent.reload.memory['last_webhook_request'].should == { "some_param" => "some_value" }
563
+        @agent.last_web_request_at.to_i.should be_within(1).of(Time.now.to_i)
564
+      end
565
+    end
518 566
   end
519 567
 
520 568
   describe "recent_error_logs?" do

+ 169 - 0
spec/models/agents/data_output_agent_spec.rb

@@ -0,0 +1,169 @@
1
+# encoding: utf-8
2
+
3
+require 'spec_helper'
4
+
5
+describe Agents::DataOutputAgent do
6
+  let(:agent) do
7
+    _agent = Agents::DataOutputAgent.new(:name => 'My Data Output Agent')
8
+    _agent.options = _agent.default_options.merge('secrets' => ['secret1', 'secret2'], 'events_to_show' => 2)
9
+    _agent.user = users(:bob)
10
+    _agent.sources << agents(:bob_website_agent)
11
+    _agent.save!
12
+    _agent
13
+  end
14
+
15
+  describe "#working?" do
16
+    it "checks if events have been received within expected receive period" do
17
+      agent.should_not be_working
18
+      Agents::DataOutputAgent.async_receive agent.id, [events(:bob_website_agent_event).id]
19
+      agent.reload.should be_working
20
+      two_days_from_now = 2.days.from_now
21
+      stub(Time).now { two_days_from_now }
22
+      agent.reload.should_not be_working
23
+    end
24
+  end
25
+
26
+  describe "validation" do
27
+    before do
28
+      agent.should be_valid
29
+    end
30
+
31
+    it "should validate presence and length of secrets" do
32
+      agent.options[:secrets] = ""
33
+      agent.should_not be_valid
34
+      agent.options[:secrets] = "foo"
35
+      agent.should_not be_valid
36
+      agent.options[:secrets] = []
37
+      agent.should_not be_valid
38
+      agent.options[:secrets] = ["hello"]
39
+      agent.should be_valid
40
+      agent.options[:secrets] = ["hello", "world"]
41
+      agent.should be_valid
42
+    end
43
+
44
+    it "should validate presence of expected_receive_period_in_days" do
45
+      agent.options[:expected_receive_period_in_days] = ""
46
+      agent.should_not be_valid
47
+      agent.options[:expected_receive_period_in_days] = 0
48
+      agent.should_not be_valid
49
+      agent.options[:expected_receive_period_in_days] = -1
50
+      agent.should_not be_valid
51
+    end
52
+
53
+    it "should validate presence of template and template.item" do
54
+      agent.options[:template] = ""
55
+      agent.should_not be_valid
56
+      agent.options[:template] = {}
57
+      agent.should_not be_valid
58
+      agent.options[:template] = { 'item' => 'foo' }
59
+      agent.should_not be_valid
60
+      agent.options[:template] = { 'item' => { 'title' => 'hi' } }
61
+      agent.should be_valid
62
+    end
63
+  end
64
+
65
+  describe "#receive_web_request" do
66
+    before do
67
+      current_time = Time.now
68
+      stub(Time).now { current_time }
69
+      agents(:bob_website_agent).events.destroy_all
70
+    end
71
+
72
+    it "requires a valid secret" do
73
+      content, status, content_type = agent.receive_web_request({ 'secret' => 'fake' }, 'get', 'text/xml')
74
+      status.should == 401
75
+      content.should == "Not Authorized"
76
+
77
+      content, status, content_type = agent.receive_web_request({ 'secret' => 'fake' }, 'get', 'application/json')
78
+      status.should == 401
79
+      content.should == { :error => "Not Authorized" }
80
+
81
+      content, status, content_type = agent.receive_web_request({ 'secret' => 'secret1' }, 'get', 'application/json')
82
+      status.should == 200
83
+    end
84
+
85
+    describe "returning events as RSS and JSON" do
86
+      let!(:event1) do
87
+        agents(:bob_website_agent).create_event :payload => {
88
+          "url" => "http://imgs.xkcd.com/comics/evolving.png",
89
+          "title" => "Evolving",
90
+          "hovertext" => "Biologists play reverse Pokemon, trying to avoid putting any one team member on the front lines long enough for the experience to cause evolution."
91
+        }
92
+      end
93
+
94
+      let!(:event2) do
95
+        agents(:bob_website_agent).create_event :payload => {
96
+          "url" => "http://imgs.xkcd.com/comics/evolving2.png",
97
+          "title" => "Evolving again",
98
+          "hovertext" => "Something else"
99
+        }
100
+      end
101
+
102
+      it "can output RSS" do
103
+        content, status, content_type = agent.receive_web_request({ 'secret' => 'secret1' }, 'get', 'text/xml')
104
+        status.should == 200
105
+        content_type.should == 'text/xml'
106
+        content.gsub(/\s+/, '').should == Utils.unindent(<<-XML).gsub(/\s+/, '')
107
+          <?xml version="1.0" encoding="UTF-8" ?>
108
+          <rss version="2.0">
109
+          <channel>
110
+           <title>XKCD comics as a feed</title>
111
+           <description>This is a feed of recent XKCD comics, generated by Huginn</description>
112
+           <lastBuildDate>#{Time.now.rfc2822}</lastBuildDate>
113
+           <pubDate>#{Time.now.rfc2822}</pubDate>
114
+           <ttl>60</ttl>
115
+
116
+           <item>
117
+            <title>Evolving again</title>
118
+            <description>Secret hovertext: Something else</description>
119
+            <link>http://imgs.xkcd.com/comics/evolving2.png</link>
120
+            <guid>#{event2.id}</guid>
121
+            <pubDate>#{event2.created_at.rfc2822}</pubDate>
122
+           </item>
123
+
124
+           <item>
125
+            <title>Evolving</title>
126
+            <description>Secret hovertext: Biologists play reverse Pokemon, trying to avoid putting any one team member on the front lines long enough for the experience to cause evolution.</description>
127
+            <link>http://imgs.xkcd.com/comics/evolving.png</link>
128
+            <guid>#{event1.id}</guid>
129
+            <pubDate>#{event1.created_at.rfc2822}</pubDate>
130
+           </item>
131
+
132
+          </channel>
133
+          </rss>
134
+        XML
135
+      end
136
+
137
+      it "can output JSON" do
138
+        agent.options['template']['item']['foo'] = "hi"
139
+
140
+        content, status, content_type = agent.receive_web_request({ 'secret' => 'secret2' }, 'get', 'application/json')
141
+        status.should == 200
142
+
143
+        content.should == {
144
+          'title' => 'XKCD comics as a feed',
145
+          'description' => 'This is a feed of recent XKCD comics, generated by Huginn',
146
+          'pubDate' => Time.now,
147
+          'items' => [
148
+            {
149
+              'title' => 'Evolving again',
150
+              'description' => 'Secret hovertext: Something else',
151
+              'link' => 'http://imgs.xkcd.com/comics/evolving2.png',
152
+              'guid' => event2.id,
153
+              'pubDate' => event2.created_at.rfc2822,
154
+              'foo' => 'hi'
155
+            },
156
+            {
157
+              'title' => 'Evolving',
158
+              'description' => 'Secret hovertext: Biologists play reverse Pokemon, trying to avoid putting any one team member on the front lines long enough for the experience to cause evolution.',
159
+              'link' => 'http://imgs.xkcd.com/comics/evolving.png',
160
+              'guid' => event1.id,
161
+              'pubDate' => event1.created_at.rfc2822,
162
+              'foo' => 'hi'
163
+            }
164
+          ]
165
+        }
166
+      end
167
+    end
168
+  end
169
+end

+ 11 - 3
spec/models/agents/webhook_agent_spec.rb

@@ -10,11 +10,11 @@ describe Agents::WebhookAgent do
10 10
   end
11 11
   let(:payload) { {'some' => 'info'} }
12 12
 
13
-  describe 'receive_webhook' do
13
+  describe 'receive_web_request' do
14 14
     it 'should create event if secret matches' do
15 15
       out = nil
16 16
       lambda {
17
-        out = agent.receive_webhook('secret' => 'foobar', 'payload' => payload)
17
+        out = agent.receive_web_request({ 'secret' => 'foobar', 'payload' => payload }, "post", "text/html")
18 18
       }.should change { Event.count }.by(1)
19 19
       out.should eq(['Event Created', 201])
20 20
       Event.last.payload.should eq(payload)
@@ -23,9 +23,17 @@ describe Agents::WebhookAgent do
23 23
     it 'should not create event if secrets dont match' do
24 24
       out = nil
25 25
       lambda {
26
-        out = agent.receive_webhook('secret' => 'bazbat', 'payload' => payload)
26
+        out = agent.receive_web_request({ 'secret' => 'bazbat', 'payload' => payload }, "post", "text/html")
27 27
       }.should change { Event.count }.by(0)
28 28
       out.should eq(['Not Authorized', 401])
29 29
     end
30
+
31
+    it "should only accept POSTs" do
32
+      out = nil
33
+      lambda {
34
+        out = agent.receive_web_request({ 'secret' => 'foobar', 'payload' => payload }, "get", "text/html")
35
+      }.should change { Event.count }.by(0)
36
+      out.should eq(['Please use POST requests only', 401])
37
+    end
30 38
   end
31 39
 end

+ 8 - 6
spec/models/agents/website_agent_spec.rb

@@ -11,8 +11,9 @@ describe Agents::WebsiteAgent do
11 11
         'url' => "http://xkcd.com",
12 12
         'mode' => 'on_change',
13 13
         'extract' => {
14
-          'url' => {'css' => "#comic img", 'attr' => "src"},
15
-          'title' => {'css' => "#comic img", 'attr' => "title"}
14
+          'url' => { 'css' => "#comic img", 'attr' => "src" },
15
+          'title' => { 'css' => "#comic img", 'attr' => "alt" },
16
+          'hovertext' => { 'css' => "#comic img", 'attr' => "title" }
16 17
         }
17 18
       }
18 19
       @checker = Agents::WebsiteAgent.new(:name => "xkcd", :options => @site, :keep_events_for => 2)
@@ -21,7 +22,6 @@ describe Agents::WebsiteAgent do
21 22
     end
22 23
 
23 24
     describe "#check" do
24
-    
25 25
       it "should validate the integer fields" do
26 26
         @checker.options['expected_update_period_in_days'] = "nonsense"
27 27
         lambda { @checker.save! }.should raise_error;
@@ -110,7 +110,8 @@ describe Agents::WebsiteAgent do
110 110
         @checker.check
111 111
         event = Event.last
112 112
         event.payload['url'].should == "http://imgs.xkcd.com/comics/evolving.png"
113
-        event.payload['title'].should =~ /^Biologists play reverse/
113
+        event.payload['title'].should == "Evolving"
114
+        event.payload['hovertext'].should =~ /^Biologists play reverse/
114 115
       end
115 116
 
116 117
       it "should turn relative urls to absolute" do
@@ -239,8 +240,9 @@ describe Agents::WebsiteAgent do
239 240
         'url' => "http://www.example.com",
240 241
         'mode' => 'on_change',
241 242
         'extract' => {
242
-          'url' => {'css' => "#comic img", 'attr' => "src"},
243
-          'title' => {'css' => "#comic img", 'attr' => "title"}
243
+          'url' => { 'css' => "#comic img", 'attr' => "src" },
244
+          'title' => { 'css' => "#comic img", 'attr' => "alt" },
245
+          'hovertext' => { 'css' => "#comic img", 'attr' => "title" }
244 246
         },
245 247
         'basic_auth' => "user:pass"
246 248
       }

+ 23 - 0
spec/routing/webhooks_controller_spec.rb

@@ -0,0 +1,23 @@
1
+require 'spec_helper'
2
+
3
+describe "routing for web requests" do
4
+  it "routes to handle_request" do
5
+    resulting_params = { :user_id => "6", :agent_id => "2", :secret => "foobar" }
6
+    get("/users/6/web_requests/2/foobar").should route_to("web_requests#handle_request", resulting_params)
7
+    post("/users/6/web_requests/2/foobar").should route_to("web_requests#handle_request", resulting_params)
8
+    put("/users/6/web_requests/2/foobar").should route_to("web_requests#handle_request", resulting_params)
9
+    delete("/users/6/web_requests/2/foobar").should route_to("web_requests#handle_request", resulting_params)
10
+  end
11
+
12
+  it "supports the legacy /webhooks/ route" do
13
+    post("/users/6/webhooks/2/foobar").should route_to("web_requests#handle_request", :user_id => "6", :agent_id => "2", :secret => "foobar")
14
+  end
15
+
16
+  it "routes with format" do
17
+    get("/users/6/web_requests/2/foobar.json").should route_to("web_requests#handle_request",
18
+                                                           { :user_id => "6", :agent_id => "2", :secret => "foobar", :format => "json" })
19
+
20
+    get("/users/6/web_requests/2/foobar.atom").should route_to("web_requests#handle_request",
21
+                                                           { :user_id => "6", :agent_id => "2", :secret => "foobar", :format => "atom" })
22
+  end
23
+end